/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package model2;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import java.util.ArrayList;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;

/**
 *
 * @author Dustin
 */
public class WebSiteGetter {

    private List<URL> fetchedURLs;
    private List<URL> toFetchURLs;

    public static void main(String args[]) throws IOException {
        if (args.length < 1) {
            System.out.println("Fout, geef een URL op als argument!");
            System.exit(1);
        }
        try {
            WebSiteGetter dingetje = new WebSiteGetter();
            dingetje.addToFetchURL(new URL(args[0]));
            dingetje.fetch();
        } catch (MalformedURLException ex) {
            System.out.println("Fout, is dat wel een URL?");
            System.exit(2);
        }
    }

    public WebSiteGetter() {
        fetchedURLs = new ArrayList<URL>();
        toFetchURLs = new ArrayList<URL>();
    }

    public void addToFetchURL(URL url) {
        toFetchURLs.add(url);
    }

    public void fetch() {
        while (toFetchURLs.size() > 0) {
            URL url = toFetchURLs.remove(0);
            if (url.toString().indexOf('#') != -1) {
                try {
                    if (fetchedURLs.contains(new URL(url.toString().substring(0, url.toString().indexOf('#'))))) {
                        // Deze hebben we al gehad, next!
                        continue;
                    }
                } catch (Exception e) {
                }
            }

            if (fetchedURLs.contains(url)) {
                continue;
            }
            fetchedURLs.add(url);

            try {
                HttpURLConnection conn = (HttpURLConnection) url.openConnection();
                HttpURLConnection.setFollowRedirects(true);
                // For some reason I have to do this to get a new location after a redirect.
                conn.getHeaderField("Location");
                URL newURL = conn.getURL();

                if (!newURL.equals(url)) {
                    if (fetchedURLs.contains(newURL)) {
                        continue;
                    }
                    System.out.println("Following redirect: " + url + " -> " + newURL);
                    fetchedURLs.add(newURL);
                    url = newURL;
                }

                int contentLength = conn.getContentLength();
                if (contentLength < 0) {
                    // Het zou netter zijn om dit anders te doen, maar goed, 't moet maar even.
                    contentLength = 100000;
                    System.out.println("Unknown content length @ " + url + ": fetching with maximum of 100000 bytes");
                    //continue;
                }
                byte[] content = new byte[contentLength];

                InputStream is = conn.getInputStream();

                int b;
                int count = 0;
                while ((b = is.read()) != -1) {
                    content[count] = (byte) b;
                    count++;
                }

                String contentType = conn.getContentType();
                if (contentType.matches(".*text\\/html.*")) {
                    // HTML
                    WebPage wo = new WebPage(url, content);
                    toFetchURLs.addAll(wo.getLocalLinks());
                    wo.writeToFile();
                } else {
                    // Binary
                    WebObject wo = new WebObject(url, content);
                    wo.writeToFile();
                }
            } catch (IOException e) {
                System.out.println("Error fetching " + url + ": " + e);
            }
        }
    }
}

